{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8924f6a1",
   "metadata": {},
   "source": [
    "## 依赖库,强化学习环境创建，参考链接https://www.bilibili.com/video/BV1S34y1d7sR?spm_id_from=333.788.player.switch&vd_source=86585d381bc8435aff1a5c7b3834ee2b&p=5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4a5ce7a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: opencv-python in d:\\python3.12\\lib\\site-packages (4.12.0.88)\n",
      "Requirement already satisfied: numpy<2.3.0,>=2 in d:\\python3.12\\lib\\site-packages (from opencv-python) (2.2.6)\n",
      "Requirement already satisfied: pillow in d:\\python3.12\\lib\\site-packages (12.0.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install opencv-python\n",
    "!pip install pillow\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "00e0a5f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''依赖库'''\n",
    "import numpy as np\n",
    "import cv2\n",
    "from PIL import Image\n",
    "import time\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import style\n",
    "style.use('ggplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d4971393",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''参数'''\n",
    "SIZE =10 #环境空间大小\n",
    "EPISODES =30000 #训练次数\n",
    "SHOW_EVERY = 3000 #每多少次展示一次\n",
    "\n",
    "\n",
    "'''奖惩'''\n",
    "FOOD_REWARD = 25 #奖励\n",
    "ENEMY_PENALITY=300# 惩罚\n",
    "MOVE_PENALITY = 1   #移动惩罚\n",
    "\n",
    "'''需要计算的参数'''\n",
    "epsilon = 0.6 #探索率\n",
    "EPS_DECAY = 0.9995 #探索率衰减\n",
    "DISCOUNT = 0.9 #折扣率\n",
    "\n",
    "Q_table ='qtable_1761747589.pickle'\n",
    "\n",
    "#三种颜色\n",
    "d = {'red':(255,0,0),'green':(0,255,0),'blue':(0,0,255),\n",
    "}\n",
    "\n",
    "PLAYER_N =\"red\"\n",
    "FOOD_N =\"green\"\n",
    "ENEMY_N =\"blue\"\n",
    "LEARNING_RATE=0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b0a188ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''创建对象'''\n",
    "class Cube:\n",
    "    def __init__(self):\n",
    "        #-----位置\n",
    "        self.x = np.random.randint(0,SIZE)\n",
    "        self.y = np.random.randint(0,SIZE)\n",
    "        \n",
    "    def __str__(self):\n",
    "        return f\"{self.x},{self.y}\"\n",
    "    \n",
    "    def __sub__(self,other):\n",
    "        return (self.x-other.x,self.y-other.y)\n",
    "    def action(self,choise):\n",
    "        if choise == 0:\n",
    "            self.move(x = 1, y = 1)\n",
    "        elif choise == 1:\n",
    "            self.move(x = -1, y = 1)\n",
    "        elif choise == 2:\n",
    "            self.move(x = 1,  y =-1)\n",
    "        elif choise == 3:\n",
    "            self.move(x = -1, y =-1)\n",
    "        \n",
    "    def move(self,x=False,y=False):\n",
    "        if not x:\n",
    "            self.x = (self.x + np.random.randint(-1,2))\n",
    "        else:\n",
    "            self.x += x\n",
    "        if not y:\n",
    "            self.y = (self.y + np.random.randint(-1,2))\n",
    "        else:\n",
    "            self.y += y\n",
    "            \n",
    "        #----限制边界\n",
    "        if self.x < 0:\n",
    "            self.x = 0\n",
    "        elif self.x > SIZE-1:\n",
    "            self.x = SIZE-1\n",
    "        if self.y < 0:\n",
    "            self.y = 0\n",
    "        elif self.y > SIZE-1:\n",
    "            self.y = SIZE-1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8f6de25c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "130321\n",
      "[-0.25401949810889857, -4.714832114092343, -4.311156712723684, -3.6236949546125734]\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "if Q_table is None:\n",
    "    # Q_table = {}\n",
    "    # for x1 in range(-SIZE+1,SIZE):#食物横坐标差值\n",
    "    #     for y1 in range(-SIZE+1,SIZE):#食物纵坐标差值\n",
    "    #         for x2 in range(-SIZE+1,SIZE):#敌人\n",
    "    #             for y2 in range(-SIZE+1,SIZE):\n",
    "    #                 Q_table[((x1,y1),(x2,y2))] = [np.random.uniform(-5,0) for i in range(4)]#初始化Q值\n",
    "    pass\n",
    "else:\n",
    "    with open('qtable.pkl','rb') as f:\n",
    "        Q_table = pickle.load(f)\n",
    "print(len(Q_table))\n",
    "print(Q_table[((3,6),(1,1))])\n",
    "print(np.argmax(Q_table[((3,6),(1,1))]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ed4f118",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "on 0,epsilon:0.6\n",
      "mean reward :nan\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995FD0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\python3.12\\Lib\\site-packages\\numpy\\_core\\fromnumeric.py:3860: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "d:\\python3.12\\Lib\\site-packages\\numpy\\_core\\_methods.py:145: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F920>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F920>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F920>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F920>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F920>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F8F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F8F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F8F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F8F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F8F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F8F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F8F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F8F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F8F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F8F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA10>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA10>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA10>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA10>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA70>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA70>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA70>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA70>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA70>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA70>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA70>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F9E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F890>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F890>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F890>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB30>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB30>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D3D0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D3D0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D3D0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FAD0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FAA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FAA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FAA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FAA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB30>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D3D0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB60>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D3D0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB60>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D3D0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB60>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D3D0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB60>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D3D0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB60>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D3D0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB30>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB30>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB30>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB30>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB30>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB30>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "on 3000,epsilon:0.1338278844804114\n",
      "mean reward :-232.422\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "on 6000,epsilon:0.029849837774170575\n",
      "mean reward :-103.268\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9961E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9961E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9961E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9961E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9961E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9961E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9963F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9961E0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996F00>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960F0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA995CA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9961B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9960C0>\n",
      "on 9000,epsilon:0.006657901069001207\n",
      "mean reward :-78.522\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996150>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996150>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996150>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996150>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996150>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996150>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996150>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996FC0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996000>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996FC0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "on 12000,epsilon:0.0014850213585738343\n",
      "mean reward :-64.41733333333333\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996FC0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA9964B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "on 15000,epsilon:0.00033122877804360457\n",
      "mean reward :-52.547333333333334\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCAFDE4B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCAFDDFA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCAFDE4B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCAFDDFA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCAFDE4B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCAFDDFA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCAFDE4B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCAFDDFA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCAFDE4B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCAFDDFA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCAFDE4B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCAFDDFA0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCAFDE4B0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996FC0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACAA996480>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "on 18000,epsilon:7.387941107434565e-05\n",
      "mean reward :-44.98133333333333\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "on 21000,epsilon:1.6478542151230624e-05\n",
      "mean reward :-37.168\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "on 24000,epsilon:3.675480725700803e-06\n",
      "mean reward :-33.1\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "on 27000,epsilon:8.19803016615111e-07\n",
      "mean reward :-28.176\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D3D0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D3D0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D3D0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D3D0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D490>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D4C0>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D430>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FA40>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59F980>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59D460>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n",
      "<PIL.Image.Image image mode=RGB size=10x10 at 0x2ACCF59FB90>\n"
     ]
    }
   ],
   "source": [
    "episode_rewards = []\n",
    "for episode in range(EPISODES):\n",
    "    #-----实例化对象\n",
    "    player = Cube()\n",
    "    food = Cube()\n",
    "    enemy = Cube()\n",
    "    \n",
    "    if episode % SHOW_EVERY == 0:\n",
    "        print(f\"on {episode},epsilon:{epsilon}\")\n",
    "        print(f\"mean reward :{np.mean(episode_rewards[-SHOW_EVERY:])}\")\n",
    "        Show = True\n",
    "    else:\n",
    "        Show = False\n",
    "\n",
    "\n",
    "    episode_reward=0\n",
    "    for t in range(200):#200步\n",
    "        #-----获取状态\n",
    "        state = (player-food,player-enemy)\n",
    "        if np.random.random() < epsilon:\n",
    "            #-----随机选择动作\n",
    "            action = np.random.randint(0,4)\n",
    "        else:\n",
    "            #-----选择Q值最大的动作\n",
    "            action = np.argmax(Q_table[state])\n",
    "        # print(state)\n",
    "        \n",
    "        player.action(action)\n",
    "        # food.move()\n",
    "        # enemy.move()\n",
    "        #-----计算奖励\n",
    "        # 奖励\n",
    "        if player.x == food.x and player.y == food.y:\n",
    "            reward = FOOD_REWARD\n",
    "        elif player.x == enemy.x and player.y == enemy.y:\n",
    "            reward = - ENEMY_PENALITY\n",
    "        else:\n",
    "            reward = - MOVE_PENALITY\n",
    "        # print(f\"reward:{reward}\")\n",
    "        #-----更新Q值\n",
    "        current_q = Q_table[state][action]\n",
    "        new_state = (player-food,player-enemy)#新的状态值\n",
    "        max_future_q = np.max(Q_table[new_state])#最大的未来期望值\n",
    "        # print(f\"max_future_q:{max_future_q}\")\n",
    "        \n",
    "        \n",
    "        if reward ==FOOD_REWARD:\n",
    "            new_q = FOOD_REWARD\n",
    "        else:\n",
    "            new_q = (1 - LEARNING_RATE) * current_q + LEARNING_RATE * (reward + DISCOUNT * max_future_q)\n",
    "        Q_table[state][action] = new_q\n",
    "        \n",
    "        \n",
    "        \n",
    "        if Show:\n",
    "            env =np.zeros((SIZE,SIZE,3),dtype=np.uint8)#10*10*3\n",
    "            env[food.x,food.y] = d[FOOD_N]\n",
    "            env[enemy.x,enemy.y] = d[ENEMY_N]\n",
    "            env[player.x,player.y] = d[PLAYER_N]\n",
    "\n",
    "            img= Image.fromarray(env,'RGB')\n",
    "            # print(img)\n",
    "            img = img.resize((800,800))\n",
    "\n",
    "            cv2.imshow(\"image\",np.array(img))\n",
    "            \n",
    "            if reward == FOOD_REWARD or reward == -ENEMY_PENALITY:\n",
    "                if cv2.waitKey(500) & 0xFF == ord('q'):\n",
    "                    break\n",
    "            else:\n",
    "                if cv2.waitKey(1) & 0xFF == ord('q'):\n",
    "                    break\n",
    "\n",
    "        episode_reward += reward\n",
    "        \n",
    "        if reward == FOOD_REWARD or reward == ENEMY_PENALITY:\n",
    "            break\n",
    "    episode_rewards.append(episode_reward)\n",
    "    epsilon *= EPS_DECAY\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7e1330ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAksAAAG0CAYAAAAikWNTAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjcsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvTLEjVAAAAAlwSFlzAAAPYQAAD2EBqD+naQAASCJJREFUeJzt3QeUE1X7x/FnZpdepPcugohUO6LyKjZAAVFEsGABC/LHrmDBggV9FQsqiigovFRFBKXYK1aQplTpsNLrUjfzP89dMiS72bAsyU7K93NOTqYluXs3m/z2zp17LcdxHAEAAEBIdujNAAAAICwBAAAcAS1LAAAAYRCWAAAAwiAsAQAAhEFYAgAACIOwBAAAEAZhCQAAIAzCEgAAQBip4XYi97Zu3SoHDx6MeJWVL19eNm7cyK+Cuow5vDepy1jFe5O6zI3U1FQpXbp07o7N1VE4Ig1KBw4ciGhNWZblPjez0lCXsYT3JnUZq3hvUpfRwGk4AACAMAhLAAAAYRCWAAAAwiAsAQAAhEFYAgAACIOwBAAAEAZhCQAAIAzCEgAAQBiEJQAAgDAISwAAAGEQlgAAAMIgLAEAAIRBWAIAAEfk7N2TtLWU6nUBAABAbHIcR3zfThPfB68f3tigidh39BOrcBH3GMuyIv/au3eJ766uZtnq/n9in91avEJYAgAAhrMxTXzP9xWrQzexm5wua265LHvN/D1HfL2vDt5WsJDYL30gVqHCYWvSycgQKyUleJsvQ5ypH4pVpYb4vv7UPH+2xw1/VZxTWx7x+aOFsAQAQIJwli0U2btHrIbNjhyK+vXMef/wVyXjaF54/z7x3dk5aJP9+ngR2xbfWy+I/Plz8L6HXxSr1gnibPpXfH17ZL5mmKe3n3zds6CkCEsAAMQ5Z/0a8T12R/DGmnVFDh4Qu99/xSpY6PCx61aJr/+dR/X8dt8XxKpTX5wDB0R27RDfAzce8TG+XlflvO/pe3P/2m99LJbtbRdrwhIAADFI+wI50z4S5/fvxb6htzhfThHnpy9z/wQrlx4xtISkrVI+n3s6zL53gAlKyipQQKR0WUkZ+knQQ3w/fSXOey9LnqUWEOuK68Wqe5JYtU8wP7scPJj5ejGAsAQAQIzw/fa9OG+/kH37U3dH/sVKlZGUF4a7q44GpL3pYhUtbta103blypVl/fr1meElDLvF+SJ68z/X2pXie7x35vP0fECsug3EKl02c5+esgsIcHavfmI1PTPo+UyH8RgJSoqwBADAMXIWLxDZt1d8Y4aK1epSsZqeIVb5Skd31VnP9sdUBuvKG8W+uKPpj+RMGStW55vE9+IjIquXZz/2+jvFPuei4G16qutQUDpWVtWa2Vqf3H0FC+W4L1YRlgAAyCPn4AHx3d4peNu4YeZmXdRR7Kty7tvj7E0XWbtKHG1N+nJy6INObCz2f9qIs3evWFVriFStJVZq5le3ae3ZvVOs4iWDHqIhzbqxj1lOeeyVzGN37hBn7q+eXn4fzwhLAAAczcCMhQqLM+I1cX78IvyxMyaKc1YrsarVDt6+e6f47uqW4+PsV/7nngrzCzWKkTlVlSUo5cQqUVIsglKeEZYAAMiBs2WT+B68KVf1Y512jlgdrzOnvXxvPmu2+Z7IbOHJLXvIxGzjEMF7hCUAAPytRjpOUb2GYhUoKM6smW7oCce+8xGxmpx+eIOeBru0kxlo8YganSr2jXeJFC8RlVGwERmEJQBAUvB98Yk4Y99x1+1n3hY5rrTpcOwsmC2+l/vn/snKlBf76bfc/kNZWe26mFNkzvj3Qu6373lKrAZNjv6HgCcISwCAhHSkK8zCjWAdyL71AZFTzj6qlh8NYNrB22mtr++IZadEbQ41RJ+3Q2ICABAFZpTqY7wU3x+ULJ2TLI8hRy/H16BklglKcYuWJQBA3PL98q0477wYtG11mOPtN7QfkSMy73fxvflc8L7b+4rV/Cxx0nebiWFzOsWG5MM7AQAQ03yf/E+cyWPMlWbWBZeLrFkuvuceyPXj7bcnZW/Vad7CHRgx6+kxq2ixyBUeCYGwBACICTrdhjN5tBmg0TrzP+J8/Wnw/okfmFtuWT3vF/u0c458HKfHcASEJQBARPh+/U4kfZc4o4aI/chLYums97lgRqJesUR8z9x3eFuWoBR2bKPON4mUKCWi/YN8GVK5WnVJS0s74nxmQG4RlgAAxyxDT4vpGEWH+AbcE7Tf7tNfrJNPCdrmmzhSnM/GHfnJ6zYQu1N38Q188PDzPfN2yLnXLJ29npYiRBhhCQBw1DJ6XH5Ux/teeUKs088T6+a7xPnhC3E+eD3HY+23PhbZsF7k4P7MudAOhZ94m3wViYOwBAA4uoljs7QaBbJaXihStJg4Mz7O/thfvzW3kI878z9iXXeHGZ/IqFSV3wpiBmEJAJArzrYt4ru/e4777fueFqt+o8yVq24S5991IqXKZM6VFnAKLZDVur3YV9/MbwAxjbAEABDf15+KVaZ80BxnzqZ/xfd8XxOCdHqPrEHJ6vmAWI1PFatQ4ZA1aFWskrlQt4E5hZahI2ZvTDOb7P+OEClW3PQxAmIdYQkAklTWudKCrh2rXU9k+eLM4x6+Ndtj7bcmuiNT51aKzsUGxCHCEgAkMN8Pn4sz4rXDG8pWELv7/4kz9zdxPp+U8wMPBaVQ7Dc/OuqgBMQzwhIAJAhnb7r43nlJZM6vOR+0eYP4Xnwk90+qYyWtXOqu2oPHMQ0Ikg5hCQBimA6s6OjpsnHDzLrV5iqxOlybbSwhRwd1fPrePL1G1kvynS2bxPfgTWI/OkisGscfQ+mBxEBYAoAY5WxME592ig7c9tl4c7OfHSpSupz4but4xOfxj6bt+DJE1q8R59Nx4mRkiKUdrLvdnu14q0w5xjQCAhCWACCfZJ2wNeQxf80W38g33avGcuLr2yP0jhrHS8qjg8TZv09k6V8i9Rq5p81MP6OqNc2caQByj7AEAFGkHalN+Nm6yd1mde8jVoMmpgUn6+X7zv/eCvk89ovvi5Q4Tnw92+f4WtZlXcS+vGvmsg7ueFKziP0cQDIjLAFAFDg7tonv3utD7xv+StBl+qsPjXzt/PB5tmP1NJnd6lJ33YxXFDjVSOXqkvJkzlOHADh2hCUAOEbO33PE99Kjx/YcWYKSniqzTzsnx0v3fXd3E6vjdWKf3+6YXhfAkRGWACCXnEXzxPffh82y/fp4kUXzTFAKO16R0lNuerukkzhjhmrnJXG+/jTkoXqlm922c9in0z5IKa+N5fcG5BPCEgCEoVeNyeL52VqOfL2uOnK9lasoKXrVWgDrmkNXt3XNHBVbO3xXLFlC1n87Q6RBM7EKHZpIFkDMICwBQA4y+t8psm5VrurHfuE9sUqVFWdPukjhIke86i3oscWKi93sLHO1HIDYY3tdAADIL2Zi2AnvZQaaUPt9PnFmzTQdqE0n6hBByX5umEihIkGTydqDRpqgZNaLFD2qoAQg9tGyBCApZLz5rMismWbZmT7xqB9vvz3JDUEpg+kvBCSThA5LvXr1ko0bNwZt69q1q3To0MFdX7lypQwbNkyWLVsmJUuWlEsuuUTat895HBMA8ceZ94cblI6W/eoY01oEIHkldFhSnTt3ltatW7vrhQsXdpfT09NlwIAB0qhRI+nRo4esWrVK3nzzTSlWrFjQYwDEN9+rTxxeqVLjiP2Q7L4viFWnfvQLBiAuJHxYKlKkiJQqVSrkvh9++EEOHjwod9xxh6Smpkr16tVlxYoVMmXKlBzD0oEDB8zNT5vl9TX8y5Hkfz76P1CXsSZe3ptO+m7J+L8u7rp9891in3V+0DG+BbNE7BSxjj8xc9TrfBYvdRkvqE/qMhoSPix9/PHH8uGHH0q5cuWkZcuW0rZtW0lJSTH7Fi9eLA0aNDBBya9JkyYyadIk2bVrlxQvXjzb802cOFEmTJjgrteuXVsGDhwo5cuXj9rPUKlSpag9d7KhLpOnPndN/VC2Dn42aFuV9leLlZLlY69yW4kFsVyX8Yj6pC4jKaHD0qWXXmrCjIaeRYsWyejRo2Xr1q1yww03mP3btm2TChUqBD3G3wql+0KFpY4dO0q7du2y/RejfaO0lSqS9Ln1Dz4tLY1LiqnLmBKL703nwH6R5UvEN+sncb74JNv+lOffk7QNwX0YY0Es1mU8oz6py9zShpLcNnTEXVgaNWqUafkJZ9CgQVK1atWgUFOzZk1TMUOHDjWdvAsUKJCn19fH5fTYaH3Q6fPyIUpdxiKv35u+72eI8/7gIx5nvzxKpFiJmP478rouEw31SV1GUtyFpcsuu0xatWoV9piKFSuG3H7CCSdIRkaGaQWqUqWKaUXSFqRA/vWc+jkBiA1Bk8nm5MTGYt/9pFg2Q8oBSKKwpJf36y0vtPO2NtH6H1+vXj1zak5Pn/n7Lc2dO9cEqVCn4AB4z/fFJHHGDgt/UKWqkvLUm/lVJAAJLu7CUm5p5+0lS5ZIw4YNzdVquj5ixAg555xz3CCkHb7Hjx8vQ4YMMWMrrV69WqZOner2aQKQP5ztW0VWLROr0ak5n6L6bLw4H4/Mts9+4DmRug24mgxA1CRsWNKWop9++smEIb3UXzty65Vwgf2YihYtKo888ogZlPKhhx6SEiVKSKdOnRhjCchHzj+LxPfs/UHbrDZXiRQtLs6E98I+1n7zI7ECrmYFgGiwHHoURoT2gwocfykS9JRh5cqVZf369XT8pC5jSqTem76JI8X5bFxSj6rN3zn1GasS/b1ZoECBxL0aDkD8y3i8t8jalUf9OPuZt8Uqz3hEAPIXYQlAvnDWLBfnq0/F+X5Gtn1W68vFvvoWcfbvE2fMUJF9+0QqVBKryeniGzZIJG2N2G9NFMvOHFAWAPITYQlA1JiO2SNeFefHL3M8xn56iFgVqphlnW7Euv7OoP0pT73BbwiApwhLAKIWlHw92+e43375f2IVY4gOALGPsATgmDlpa8X36O1HPM7qcZ9IRoZYZ7biUn8AcYOwBCDPnG2bJeO+7kc8jo7ZAOIZYQnAUXN8GbK6begBJLOyzm7NFWwA4hphCcBR8036X7ZtVtfbxGpxvliFClOjABIKYQnAsU1g2/g0Sen9KLUIIGERlgDkeQJb+76nxarfiBoEkNAISwCOvjVJRI67qY/sPrFxQk6DAACBCEsADGfpX+J741mRndvF6vmAOD9/LTL3t5C1kzJ4nJSsXUd2r19P7QFIeIQlIMn5vvlMnFFDgrY5bz8f8li7/ytiVavNGEkAkgphCUhSzp508f1fl1wfbz/5hliVq0W1TAAQiwhLQJJx/vpTnNX/iDNheMj91pn/yTwFpwHp8cEixYqJVapsPpcSAGIHYQlI8o7aymrVRuxutx3ecPPd+VcoAIhxhCUgCTgHD4jv9k4h99nPDhWrXMV8LxMAxAvb6wIAiC5n/qwcg1LK0E8ISgBwBLQsAQnG2bVDnFk/iTP6bZGDB7Pttx8ZJFbN4z0pGwDEI8ISkACc9avF91iv8AfZtqS89XF+FQkAEgZhCYhzjs93xKBkXXaN2Jdfk29lAoBEQlgC4oCzern4nuwj1o13iXXaOeL8+bNYDZuLOI747uqa4+PsQSPFKl4yX8sKAImGsATEAQ1KynnvZXMzyzl02AYARBZXwwExLuPVJ3N1nP3SyKiXBQCSES1LQAzxjRkqzpeTj+5Bx5WWlP+OiFaRACDpEZaAGODs2Ca+e68Pe4z96CCxamRe8u/s2yuycb3Irp0itermUykBIDkRlgAPOXv3iK/31Uc8zh74rlhlyrnrVqHCItVqR7l0AADzGUw1AN7wTf0w56BUp37mfdHimaNsBwQlAED+omUJ8CAkOR9l72NknXeJWN1uF8uy+J0AQAwhLAH5wNFpR3bvFN+DN4tkhJiC5O1JhCQAiFGEJSCCfF9NMQMg2Re0M+vO3nSRzRvF93jv0A8oX0lSnnmb3wEAxDDCEhABji9DnGGDxPn1O7OeMSZ8ALIfel6kQhWRYsWpfwCIcYQl4BhDku/Wjkf1GB080irBFCQAEC8IS8Ax8OVydG3rjPMy53VLSaG+ASDOEJaAPHDSd4mvT/YJbO3B48UqVMgMMulMHiPWld0zx0QCAMQtwhJwFJwtGzOvaMvC7vWwWE3PcNetkqXE6nYbdQsACYBBKYEwnFXLJKPH5ebmzPk1ZFBSgUEJAJBYaFkCQnAO7BffHVcGbfMNHpDtOPuRQWLVzJyvDQCQmAhLgIYjHQvpodCtRqHY9z8rUqyEWFVrUH8AkOAIS4C2Gh1FUNK52gAAyYOwhKTnLF8cug5SUsV+Y4JYNl37ACCZEZaQ9HzvvXK4Duo3Erv3YyIFCzJXGwDAICwhaTnpu8WZ+5vI+tVm3brgMrG79PC6WACAGENYQtJxNqwX33MPiOzcHrRdB5AEACArwhKSju/hW0Nut1IL5HtZAACxj7CEpHLwqbuzbbN6PiD2aS09KQ8AIPYRlpAUfDM+ltXjhgVts047R+ye93tWJgBAfCAsIeH5vvhEnCxByX7+PbFKl/WsTACABAtLb7zxxlE/sWVZcvvtt+elTEBEOWPfCVq3Wl1KUAIARDYsLViwINu2/fv3y44dO8xysWLFzP3u3bvNfcmSJaVQoUK5LwUQpaEBJG2Nu16gTj3xPfi8CINMAgAiHZZef/31oPU1a9bIgAEDpGPHjtKmTRsTjpSGp08//VS+++47eeihh46mHEBEOPv3ie+pu0TS1mbbV/GVkZL277/iOA61DQDItTzN4/Duu+9K06ZNpUuXLm5QUrp8zTXXSJMmTeS9997Ly1MDx8TX66qQQcm+vCvTlgAA8i8sLVmyROrUqZPj/tq1a8vixTnMtwVEmLMnXXzj35WMHpfneIx1WRfqHQCQf1fDFS9eXGbPni0XXXRRyP26z9+PCYg23+CnRBYH96uzn3xDrMrVgi44AAAg38JS69atZdy4cfL888/LJZdcIpUqVTLb09LSZOrUqSYsde7cOU8FAo5a1qD00kixShw+PQwAQL6HpU6dOsmBAwdk8uTJ8scffwTtS0lJkQ4dOphjgGhxZs0UZ8UScaZOCNpu3/8sQQkAEBuDUmrnbr0Sbt68ebJx40azrXz58tKoUaOgTt9ApPimfijORyNy3J8y9BMqGwDgfVjat2+fPPbYY3LBBReYPktnn3125EsFBHC2bRHf/d3D1on9XPDAkwAAeHY1nA42uWHDBjrMIl84G9aFD0rVaov91sdila3AbwQAEDun4XSMpTlz5siFF14Y+RIBAXzPPZitPuyX/ydWseLUEwAgdsdZ0s7b69evl9dee00WLlwoW7ZskV27dmW7AcfCjLS9c/vhN+vLo8R+exJBCQAQ+y1L9957rzvtyQ8//JDjcWPHjs17yZD0nFFvunVg9+onVrESSV8nAIA4GjqAQf4QDc7c38T32lPZtltNz6TCAQDxE5YYcBLROu0WKigBABCX4ywBkeb88m22bfbDL4pV6wQqGwAQn2FJO3cvX75c0tPTMzvjZnHllVcey9MjiTh794gz7CV33ep0g1gXX8HpXgBAfIYlvdLt2WeflaVLl4Y9jrCE3PL1vtpdtrreJvZ/2lB5AID4DUsffPCBrFq1Svr06SN169aV3r17y8MPPywVKlSQKVOmyJIlS6Rv376RLy0SkpOREbROUAIAxP04S7Nnz5bWrVtLixYtpEiRImabXh1XqVIlueWWW8wcccOHD490WZGgfLd1dJftOx/xtCwAAEQkLO3evVuqV69ulgsXLmzu9+7d6+5v3LixGeEbCMfZvVMy7rkueOPJp1BpAID4Pw1XpkwZ2bZtm1kuUKCAlCxZUlauXCmnnXaa2aYjekd7HKaPPvpIZs2aJStWrJDU1NSQLVmbNm2SoUOHyoIFC0yoO++886Rr166SkpLiHqP73n//fVm9erWULVvWjCHVqlWrqJYdmXx3dcs+jUnA7wYAgLgNSw0aNJC5c+fKFVdcYdb1dNykSZPEtm3x+Xzy2WefSZMmTSSaDh48KGeeeabUq1dPvvrqq2z7tRzaCb1UqVIyYMAA2bp1qwwePNgEJQ1MSicEfu6558wcd9rvav78+TJkyBDzGJ3/DtHhbN4gvoduCdpm36EjdDPfGwAgQcJSu3btTFg6cOCAaVm66qqrzNQn/ulNNEzddNNNEk3+gTG/+eabkPv1NKCW6dFHHzXhp1atWnL11VfLqFGjzGO1NWrGjBmmU/r1119vHlOtWjUzHMKnn35KWIoSx5eRPSjd85RYDaIbrgEAyNewVKNGDXPzK168uAkl2pdJW5f8nb69tHjxYlNGDUp+2lr0zjvvmFNutWvXNlftNWrUKOhx2iIWrnO6BkS9+enpxsBO7pHkf75Emlom49bDnbn97JOi34qXiHXpJeqTuoxVvDepy5gJS/v27ZNChQpl216sWDGJFdqnKjAoqeOOO87d57/3bws8Zs+ePbJ//34pWLBgtuedOHGiTJgwwV3X0DVw4EBzBWC06FWGicA5eFDWZNlW7olXpEjlyvlWhkSpy1hBfVKXsYr3JnXpeVjq3r27CQknnniiOeWm9yVKHPuM8HqKTPs+hTNo0CCpWrWqeKVjx47mNGTW/2I2btxo+lFFkn84hrS0tJAjpMebjAmHW+xSBgwRq1JV0di6bf36qL92otWl16hP6jJW8d6kLnNLu+PktqEjT2FJ+/5o356vv/7a9O9RGmA0OPnDU7ly5Y76eS+77LIjXolWsWLFXD2XtiplHWF8+/bt7j7/vX9b4DF6Wi1Uq5LSPlp6CyVaX8L6vPH+Be/sTRdn2oeHN1Ss4snPlAh1GUuoT+oyVvHepC4jKU9hqUOHDu6bUUfy/vvvv014+uOPP+SLL74w+zQsvf7660f1vDoEgd4iQa+S0+EFNPz4T7Vpp3QNQtqRW51wwglmgM1Aeow+FpGT8cYzIrN/dtetSztRvQCAxB6UMrC5s2bNmmY074svvlguuugiqXyo/4mOcRRN+vw6xpLe6zABuqw3/+CY2lFbQ5EOF6Db//zzTxkzZowpp79lSMurwweMHDlS1q5dK9OnT5eZM2dK27Zto1r2ZOLs2BoUlJR9xQ2elQcAgKNlOXk4J5Geni6LFi1yW5SWLVsmGRkZZlTv+vXry0knnWROxengldGirVbffvtttu39+/eXhg0buv2I9Oo3HXhSO6TroJTdunXLNijliBEjzDADxzIopb5W4FVykaBhVMPn+vXr4/bUUcbtV+igWO66/fp4sQpmvzgg2hKhLmMJ9Uldxirem9RlbmnDSW77LOUpLF1zzTXmvk6dOm4fJb3pEALJirCUnbMnXXz/18VdTxn6iXiFD1DqM1bx3qQ+Y1WivzcLHEVYytNpOG2l0VNf2h9IL7/X+x07duTlqZDAAoOS/eBAT8sCAEC+dvB+7733zFxwegpOT8WNGzfOhCbtnK2n4fytTccff3yeC4b4lvW/EKtuA8/KAgBAvoclbZrT6UP0dskll5htOn7NvHnzzLxwv/32mzlGO1QjSS392120B4/3tCgAAOR7WPLTK890WhFtXdKbjmuknZx1yhMdtBLJ26rke/4hd90KMdo7AAAJHZbef/99E470knztu6QDOOqYRZdffrk5BafjFIWaDgXJwdezvddFAADA27Ckl+xr36QWLVqYcKStSIGX4wN+9tvhp68BACAhw9KwYcMiXxLEPWf5YvE9c5+7bvd+1J07DwCApOyzpP2Tli9fboYO0JamSE1Vgvjke/b+oHWr8WmelQUAAM/Dkl71Nn78eDOat3r00Ufl5JNPNuMt3X333Wak7PPPPz9iBUUcCBwuoGZdL0sCAEDE5GlQyq+//tpMEdK0aVO5/fbbg/Zp65JON/LTTz9FqoyIM3avhyXlkZe8LgYAAN6FpSlTpsipp54qffr0kVNOOSXbfp0GZfXq1ZEoH+KEsytgBPc69bwsCgAA3oclHYCyWbNmOe7XOeJ27dp1LOVCnPHdfe3hleLHeVkUAAC8D0tFixYNOxfcmjVrpFSpUsdSLsQR3+fBwwNYdp7eVgAAxKQ8fatpq9KXX34pu3fvzrZPT7/pvlCn55CYnHGHh5KwXxjuaVkAAIiJq+G6dOkiDz/8sNx7771uKPrmm2/kq6++kl9++UVKly4tV155ZaTLihjkbN7oLls33yNWqTKelgcAgJgIS2XKlJHnnntORo8e7V719v3330vhwoXl7LPPNsMGMOZScnD+mu0uW6ef62lZAACIibCkA1HOmTNHypcvL7fddpu5af8lnSNOA5JOoovkmTDXeX+wu05fJQBAIjrqZJOamiovvfSSLFq0yN2mIUk7dBOUkoezZRMT5gIAksJRhyWd66ty5cqyc+fO6JQIccH34E1B6/Yroz0rCwAA0ZSnc2YdO3aUadOmybp16yJfIsSfpmeIVbSY16UAACB2OngvXrxYSpQoYa6GO+mkk0z/pYIFC2ZrgbrxxhsjVU7EEGfeH+6ydeNdYrdgDkAAQOLKU1iaPn26uzx//vwcjyMsJSbfmLfdZYISACDR5SksjR07NvIlQfzYsD7zvtmZXpcEAICo4zp/HJWMfj0Pv3lataH2AAAJj7CEXHO2bxXZmOauWyc1pfYAAAmPsIRc8w1/1V22ut1OzQEAkgJhCbn3z6GBSIsUE7vVpdQcACApEJaQK87BgyLpu8yyRVACACQRwhJyJ23N4eUiDEAJAEgeeRo6QO3du1e2bt0q+/fvNwNSli5dWgoXLhzZ0iFm+Cb9z122LurgaVkAAIjZsLRr1y6ZMmWKzJw5U9LSDl8V5VexYkU566yzpF27dmaEbySQP392F62UFE+LAgBATIalDRs2yOOPP25ak04++WRp0aKFaU0qUKCAHDhwwGxfunSpfPLJJ/Ldd9+ZYzU8If75fvjcXbZOO8fTsgAAELNhafjw4eb+xRdflCpVquR4nE6uO2DAABkxYoQ88MADkSklPOV8M9Vdtm65x9OyAAAQsx28FyxYYE6vhQtKSve3bdvWHI8EsXKpu2jZnIIDACSXXIcl27YlIyMjV8fqcXo8EovV5iqviwAAQL7LdaJp1KiRTJ48Wf7555+wx+l+Pa5x48aRKB885uzd4y5brdt7WhYAAGK6z1L37t3liSeekL59+0rdunWlTp06poN3amqqHDx40HTw1qCknby1Y/cNN9wQ3ZIjf2wKmAuuRElqHQCQdHIdlsqUKSMDBw6U6dOnyy+//CJfffWVCUnuE6WmSo0aNaRr165y8cUXM+ZSgvC9NsDrIgAAED/jLOmgk+3btzc3x3HMuEv+QSmLFy8ulmVFr6TwxpaN1DwAIKnleQRvDUYMPJnYnICr4Ow7H/G0LAAAxE1YSk9Pl1mzZsny5cuzTXdSq1Ytad68uRQrxtxhicA3IGBMpeNP9LIoAADER1jS0bk//PBDMy+cv2XJP4L3zp07zam5QoUKyRVXXCEdOjB/WCKxitO5GwCQnHIdlqZNmyajRo2Sli1bmg7cejWcdur2087ey5YtkxkzZsjo0aNNaLr00kujVW5EWcaLh0+72Q8OpL4BAEkr12Fp6tSpcu6550qvXr1CP1FqqtSvX9/ctNVJwxVhKY4tnOsuWnUbeFoUAADiYlDKTZs2SYMGufvSPOmkk8zxiP+O3VKmnJdFAQAgfsJShQoVZM6cObk69s8//zTHI/47dtsPcAoOAJDcch2WLrvsMvn555/l+eefN2Fox44dQft1Xbfrfh20Uo9H/LPKlve6CAAAxEefpfPPP99MkDtmzBj5448/zDadLNc/3YnP5zPbdHDKm2++2RyP+OMbM9Rdtl8e5WlZAACIu6EDLrzwQjnvvPNkwYIFZpylbdu2ueMslSpVyoyzdPLJJ5t1xCfny8nuslWshKdlAQAgLgel1CDUrFkzc0MCa97C6xIAABDf053owJRZR/DWueMQv5ydh/uh2deHHiICAIBkc1RhSSfOnTJlisycOVPS0tKy7a9YsaKcddZZ0q5dO+aNi0PO/4YcXila3MuiAAAQf2Fpw4YN8vjjj5vWJO2X1KJFC9Oa5J/uRLcvXbrUTIny3XffmWM1PCF+OHN/c5d1YFEAAHAUYWn48OHm/sUXX5QqVarkeNy6detkwIABMmLECHnggQeo4zjhHDggsn+fWbbaXe11cQAAiL9xlvQKOD29Fi4oKd3ftm1bczzih/PB6+6ydfq5npYFAIC4DEs6ppKOs5Qbepwej/jhpK1xl63K1T0tCwAAsSTXiaZRo0YyefJk+eeff8Iep/v1uMaNG0eifMgHji9DZPlis2zddDd1DgBAXvosde/eXZ544gnp27ev1K1bV+rUqWM6ePtH8NYO3hqUtJO3duy+4YYbcvvU8Jjv1o7uslWtlqdlAQAgbsNSmTJlZODAgTJ9+nQz99tXX31lQpL7RKmpUqNGDenatatcfPHFjLkUrwhLAADkfZwlHXSyffv25uY4jhl3yT8opc4Jx+Xm8ccJCLxW9z78DgEAiNQI3hqMSpRg7rC4t2Nb5r1ti3VWK69LAwBA/IclbVHS0286GKWfDkr5999/S3p6utSuXZvBKOPJlo2Z96XLiWWneF0aAADiOyyNGTNGpk6dKvv27ZP69etLnz59zPJTTz0lmzdvdlucLrnkEtMhHLHP+fW7Qws+r4sCAEB8h6Vvv/1WJk6cKM2bN5fy5cvLN998I2+88YbZ17RpUznzzDNN/6Uvv/zSBCq9Wu7ccxncMNY5X3+auUCrEgAAxxaW9Cq4U089Ve6//36zrmHozTfflJYtW0rPnj3d4/SYfv36mdBEWIptekrVz2p9uadlAQAg7gelXL9+fdBAkw0bNjT3p5xySrZjzzjjDFm5cmWkyohoWTzfXbTOvYR6BgDgWMKSz+eTlJSUoGEEVMmSJbMdq8MIaKdvxDbfjI/dZSugwz4AAMhDWNLRujdt2uSuFypUSK655hqpVKlStmP1uFAhCrHD2b1TZO5vmSvNW3hdHAAA4r/PkvZRWrRokbuuA1F26NAh5LF//vmnGUIAscv560932WrQxNOyAACQEGHpuuuuk507dx7xuB07dphhBUL1ZUIMWbHEXbTqn+xpUQAASIiwpKfh9HYkevotP8ZY+uijj2TWrFmyYsUKMy/d8OHDsx3TuXPnbNt0bKizzz7bXV+wYIG8//77snr1ailbtqx06tRJWrVqlfBXwTmB/ZUqV/e0PAAAJOR0J17TUcR1bKd69eqZSX1zcscdd5hxoPyKFi3qLm/YsEGee+45ufDCC6V3794yf/58GTJkiJQqVSroMQln56EpTgAAQPTC0po1a8zAlP/++6/s3r07aMwe/0jejz32mESLv9VIyxCOhiMNP6HMmDFDKlSoINdff71Zr1atmixcuFA+/fTThA5LzqIF7rL9+nhPywIAQEKGpe+++86M3q1DCVSpUsUMFZBV1vDklWHDhslbb71lQpG2IP3nP/8xQU4tWbJEGjVqFHR8kyZNQp7S89MhEQKHRdDnKlKkiLscSf7ni+TzOj6fOG8/767bhTKHgEh00ajLZEZ9UpexivcmdRkzYWn8+PHmare+ffvG9BAB2vp08sknm2EO5syZY4LT3r17pU2bNmb/tm3b5Ljjjgt6jK7v2bPHTN2iV/xlpVO+TJgwwV3Xehg4cKCZAiZaQg3PkFcH1q6UtEPLVqFCUrlyZUkmkaxLUJ+RxHszsqhP6tLzsLRlyxa57LLLIh6URo0aJZMmTQp7zKBBg6Rq1aq5er4rr7wyKNTopL+TJ092w1JedOzYUdq1a5ftv5iNGzeaflSRpM+tf/BpaWkRa6nL+HiMu2w/N8yMzJ4MolGXyYz6pC5jFe9N6jK39OKw3DZ05Cks1axZ0wSmSNMAdqQr0SpWrJjn5z/hhBPkww8/NKfRChQoYPoybd++PegYXdfTaqFalZQ+Tm+hROtL2Fy9FqHnDrwKToqXTLrgEMm6BPUZSbw3I4v6pC4jKU9hSTtEawtPs2bNzJhKkaItVdE8rafDDBQrVswNOxqeZs+eHXTM3LlzzRV2Cevgof5WjU71uiQAACRuWNJTZXqVmV7tpleQlStXTmzbztYU+sADD0i06JQqu3btMvc6b50GIaWnWnTeut9//920Emkg0lYiDUHa30hbr/wuuugimT59uowcOdJ0/NahA2bOnCkPPfSQJKyyFUQ2bxD7otCjrwMAgAiEpVWrVpl7DUnaYVqHEcgq2lcdjR07Vr799lt33R/M+vfvLw0bNjTnIjUIjRgxwjTHaojSFrELLrjAfYxeIafBSI/57LPPzKCUt912W8IOG+D4MkxQMkqV8bo4AADEBcuhA0dEaAfvwCEFIkEDp16tpp2wI/FrchbNE99/HzbL9psfipUauu9VIop0XSY76pO6jFW8N6nL3NIuObnt4B187gwJzfdyf3c5mYISAACeTneiYxKlp6eH/G9dT9MhhviHNihYyOuSAACQ+GFJpwqZMmWKme4kJ9qvCLHB2RkwRELDZl4WBQCAuGLnNSjpaNjaabpLly5mW9u2baVDhw5m7KJatWrJ7bffHumy4lhs2egu2j2jd5UiAACJJk9hadq0aWYOtX79+knr1q3NtubNm8s111xjxl/SU3M7d+6MdFlxLNJ3Z95Xri5W6jGffQUAIGnkKSzpqbdTTjnFLOtkuso/1YeOv3T++eeb1ifEkL17Mu+LFPW6JAAAJH5Y0kCUkZHhLuugjzo4pJ9OF6KT1CJ2+H7+JnPhULgFAABRDEvVq1eXlStXuus6Pcjnn39u5ovT0PTFF18k3Wz2MW/OL5n3NmEJAICoh6VzzjlHVq9e7Q7CeNVVV5lRvLVTd69evWTdunVux294zwzrcKgl0L70Sq+LAwBAXMlTT1+dR01vfieeeKK89NJL8scff5g54ho3bixVqlSJZDlxDJzffzy8UidyEx8DAJAMInZZVMWKFaVNmzaRejpE0v697qJFB28AAPIvLC1evFgWLFgg27dvl4svvtj0U9q3b5+sXbvWtCwVLlz4WJ4eEeIMf9XcW6edQ50CAJAfYUmHCXj55Zflt99+c7edeuqpJizpJIZPP/20GaTyiiuuyMvTI4KclcsOLwcMTAkAAKLYwXvMmDGmf1KPHj1MaAqkwwiceeaZQUEK3nFWHQ5Ldrur+VUAAJAfYenHH3+Uiy66yIzeXbx48Wz7q1atKhs2bMjLUyPCnNFvZy4cf6JYJ2cOJAoAAKIclnbs2CE1atTI+Ult2/RdgrecNctFDuzPXAmcSBcAAORansJS2bJlTSfunCxatMhMsguPbd3sLlqXd/W0KAAAJFVYatmypRmlW6+Gy0q3z5w5U84999xIlA/HwFkw2122zziPugQAIL+uhtOr3JYsWSL9+/c3/ZPUiBEjZNeuXWbKk2bNmkm7du3y8tSIpL3p1CcAAF6EpdTUVOnXr598//338vPPP4vP5zPDCdSsWdNMc6KtSjqEALzl/Phl5kLl6vwqAADI70EpNQxpKOJ0W2xyDs0FZ9h5OtsKAADy2mcJceDPn91F+94BnhYFAICkbFlauHChfPXVV2Y8pd27d2fObJ+l5emFF16IRBmRB74hAw//LkocRx0CAJCfYWnKlCnywQcfmNG6dQ64UANTAgAAJG1Y+uSTT+TEE0+UBx98UIoWLRr5UuGY+D4d5y7bvfpRmwAA5HefJR2dW8daIijFJufjke6y1fRMT8sCAEBShqWGDRvKqlWrIl8aRFbRYtQoAABehKWbbrpJ5s+fb07H6UCUiDFFMkOS1aWn1yUBACA5+yyVK1dOWrdubTp5jxo1ynT01slzs9JRveGBlMzfhVW9NtUPAIAXYWns2LHy0UcfSZkyZeT444+n71IMcXSKk107M1fKVvC6OAAAxL08haXPP/9cmjdvLvfff3/IFiV4aMlf7qJVhCsVAQA4VnlKOjoPnIYlglLs8U370OsiAACQUPIUljQo/f3335EvDY7d4gXUIgAAXoelq666StauXSvvvPOO/PPPP7Jjxw5zVVzWG7xjtW5P9QMA4FWfpbvuusvcr1ixwvRfCtcRHPnHObBfJ+UTcRyxLr2CqgcAwKuw1KlTJzNRLmLM5g0mKEmhIiIlSnldGgAAkjcsde7cOfIlwbHb+G/mffEShFkAACKE6/4TiO/9wYdbmAAAQEQQlhLJts1elwAAgIRDWEoQjvZV8qtT38uiAACQUAhLiSJ9t7tod7jW06IAAJBICEuJYusmd9Fq0MTTogAAkEgISwnCmfWT10UAACAhEZYShb/LUqmyHhcEAIDEQlhKEM6UMebeanWp10UBACChEJYSgLM3/fDy0r88LQsAAImGsJQAnDm/uct2p+6elgUAgERDWEoE8/9wF61qtTwtCgAAiYawlACcn7/xuggAACQswlIisfl1AgAQaXy7JhCrSw+viwAAQMIhLCUQqyr9lQAAiDTCUpxz9u09vFKjtpdFAQAgIRGW4pzz/uuHVwoV8bIoAAAkJMJSnHN+/dZdtizL07IAAJCICEtxzjrlbK+LAABAQiMsxTnnjx8zF8pV9LooAAAkJMJSHHMcx122Lr7C07IAAJCoCEuJMs3J2a09LQoAAImKsBTHfK8+6S5bBQp4WhYAABIVYSmeVa7udQkAAEh4hKV47q+0frVZtq67w+viAACQsAhLccr5bPzhFTvFy6IAAJDQCEvxavMGd5HO3QAARA9hKU45389wlxm5GwCA6CEsxbva9bwuAQAACY2wFK/qNzJ31oXtvS4JAAAJjbAUrxbNM3dWkaJelwQAgIRGWIp7ltcFAAAgoRGW4nxOOKle28uiAACQ8AhL8WjHtsx7yxYpVtzr0gAAkNBSJQ5t2LBBPvzwQ5k/f75s27ZNypQpI+ecc45cccUVkpp6+EdauXKlDBs2TJYtWyYlS5aUSy65RNq3D+4QPXPmTBk7dqxs3LhRKlWqJN26dZPmzZtLTNuYlnlfppxYqcwJBwBANMVlWFq3bp05FdWzZ08TcFavXi1vvfWW7N27V66//npzTHp6ugwYMEAaNWokPXr0kFWrVsmbb74pxYoVk9atW5tjFi1aJK+88op07drVBKQffvhBXnjhBRk4cKDUqFFDYpXv1SeyDUwJAACiIy7DUtOmTc3Nr2LFiiZAzZgxww1LGnwOHjwod9xxh2ltql69uqxYsUKmTJnihqXPPvvMPM/ll19u1rt06SLz5s2TadOmmSAWyoEDB8wtcEDIIkWKuMuR5H++bM+7Jz3bMchjXSKy701Qlx7jvUldRkNchqVQtCWpePHD/XcWL14sDRo0CDot16RJE5k0aZLs2rXLHKvHtGvXLuh59Jjffvstx9eZOHGiTJgwwV2vXbu2aYkqX768RIu2nvkdWLVcDp2EkxKdrpNSlStH7XUTUWBdgvqMJbw3qc9YVYnPzcQIS2lpaTJ16lS57rrr3G3al6lChQpBx5UqVcrdp2FJ74877rigY3Rdt+ekY8eOQQHL/1+M9nnSlqxI0ufWN6n+fP4r4A4OuM/dn97sbNmzfn1EXzNRhapLUJ+xgPcm9RmrEv29mZqamuuGjpgKS6NGjTItP+EMGjRIqlat6q5v2bJFnn76aTnrrLPc02vRVKBAAXMLJVpvJn1e97mLlTi8o2KVhHwDR1NQXYL6jCG8N6nPWOXwuRlbYemyyy6TVq1ahT1G+ycFBqUnnnhC6tevn62PkbYiZW0h8q/7W5j0fvv27UHH6Lp/fyyyKlUVZ/F8sdpc5XVRAABICjEVlvTyfr3lhj8oaZ8h7cRt28FDRtWrV09Gjx5tTo35+y3NnTtXqlSp4vZt0mO0Q3fbtm3dx+kxJ5xwgsQqZ8Oh027lDodGAAAQPXE5KKUGpccff1zKlStnrn7bsWOHaTUKbElq2bKlCUlDhgwxQwv89NNPpl9TYH+jNm3ayJw5c2Ty5Mmydu1aGTdunBmTScdjilkL55o7i8EoAQBIvpal3NLWH+1wprfbbrstaJ8GHlW0aFF55JFHzKCUDz30kJQoUUI6deoU1K9JT9/93//9n4wZM8a0QlWuXFnuv//+mB1jydm/7/BKtVpeFgUAgKQRl2FJ+zUdqW+Tqlmzpjz55JNhj9GO4XqLq2lOVHmGDAAAID/E5Wm4pOUPS2XKMxggAAD5hLAUj2GpRPDYUAAAIHoIS3HE2bo5c6F4wFhLAAAgqghLccSZ8G7mwoLZXhcFAICkQViKJ0WKZd6fcJLXJQEAIGkQluJJpWrmzjrzyFcCAgCAyCAsxZNlCzPvUwt6XRIAAJIGYSmeHJoAltG7AQDIP4SleHJojjspVcbrkgAAkDQIS3HCSVsjsm9v5krBwl4XBwCApEFYihO+J/ocXqFlCQCAfENYihcHD7iLVpGinhYFAIBkQliKM9ZZ53tdBAAAkgphKd5Ur+11CQAASCqEpTjgBJ6CY0BKAADyFWEpHuzYnnlv2yLFmEQXAID8RFiKBzu2Zd6XKCWWBiYAAJBv+OaNA86WjZkLBQp4XRQAAJIOYSkO+N54JnNh079eFwUAgKRDWAIAAAiDsBQHrIbNM+9vDBjFGwAA5AvCUhxwtm8x9xZXwgEAkO8ISzHOObBfZM2KzJWSpbwuDgAASYewFOMObkg7vFK1ppdFAQAgKRGWYtzOCSPcZatgIU/LAgBAMiIsxbjdMyZ5XQQAAJIaYQkAACAMwhIAAEAYhCUAAIAwCEsxLqV8JXNv397X66IAAJCUCEvxolQZr0sAAEBSIizFMMdxJGPr5syV40p7XRwAAJISYSmW7UkXOXggc5nRuwEA8ARhKZZtO9SqVLCQWAUKel0aAACSEmEphjnbt2YupKR6XRQAAJIWYSmWHeqvZNWq63VJAABIWoSlGObM/jnzfs9ur4sCAEDSIizFsv37Mu/prwQAgGcIS7HMssyd3ewsr0sCAEDSIizFMGf+H5n3m/71uigAACQtwlI8nY4DAAD5jrAUw6N3+9mXd/W0LAAAJDPCUqzat/fwcrHiXpYEAICkRliKVem7Mu9TU80I3gAAwBuEpVi1a4e5s0uWEuvQVXEAACD/EZZi1d69IkWKSgoT6AIA4CkmHYtRVr2GkvraWKlYobykbdjodXEAAEhatCzFOItJdAEA8BRhCQAAIAzCEgAAQBiEJQAAgDAISwAAAGEQlgAAAMIgLAEAAIRBWAIAAAiDsAQAABAGYQkAACAMwhIAAEAYhCUAAIAwCEsAAABhEJYAAADCSA23E7mXmpoal8+dbKhL6jNW8d6kPmNVaoJ+Bx3Nz2U5juNEtTQAAABxjNNwMWzPnj3y4IMPmntQl7GE9yZ1Gat4b1KX0UBYimHa6Ld8+XJzD+oylvDepC5jFe9N6jIaCEsAAABhEJYAAADCICzFsAIFCsiVV15p7kFdxhLem9RlrOK9SV1GA1fDAQAAhEHLEgAAQBiEJQAAgDAISwAAAGEQlgAAAMJIzAlfEsC0adNk8uTJsm3bNqlZs6bcdNNNUrduXUlm48aNkwkTJgRtq1Klirz88stmef/+/fL+++/LTz/9JAcOHJAmTZrILbfcIqVKlXKP37RpkwwdOlQWLFgghQsXlvPOO0+6du0qKSkp7jG6T59n9erVUrZsWenUqZO0atVK4tlff/0ln3zyiRnkdOvWrXLffffJ6aefHjSQn9bvl19+Kbt375YTTzzR1F3lypXdY3bt2iXvvvuu/PHHH2JZlpxxxhly4403mnr0W7lypQwbNkyWLVsmJUuWlEsuuUTat28fVJaZM2fK2LFjZePGjVKpUiXp1q2bNG/eXBKpPl9//XX59ttvgx6j78eHH37YXac+M02cOFF+/fVXWbt2rRQsWFDq1asn1157rfnb9svPv+14/uzNTV0+/vjj5v0bqHXr1tKzZ093nboMQeeGQ2z58ccfnWuuucb56quvnNWrVztDhgxxunfv7mzbts1JZmPHjnXuueceZ+vWre5t+/bt7v63337bue2225x58+Y5y5Ytc/r16+c88sgj7v6MjAzz+CeffNJZvny5M2vWLOemm25yRo0a5R7z77//Otdee60zYsQIU/dTp051rr76amf27NlOPNOfdfTo0c4vv/ziXHXVVeY+0MSJE50bbrjB+fXXX50VK1Y4AwcOdHr16uXs27fPPebpp5927rvvPmfx4sXO33//7fTu3dt5+eWX3f27d+92brnlFueVV15xVq1a5fzwww9Ot27dnM8//9w9ZuHChaY+J02aZOpXy9SlSxdn5cqVTiLV5+DBg019Bb5Xd+7cGXQM9ZlpwIABztdff23eM/p3+cwzzzi33367s2fPnnz/2473z97c1GX//v3NzxX43tS/XT/qMjTCUgzq27ev88477wS9eXv27Gm+0JI9LOmXdSj6x65fujNnznS3rVmzxnyRLVq0yKzrB2jnzp3Nh4Pf9OnTneuvv945cOCAWf/ggw/Mh26gQYMGmQ+hRJH1y93n8zk9evQwASawPrt27WoCj9IvDn3c0qVL3WP0S0brc/PmzW5d6heLvy7VyJEjnT59+rjrL730kvPss88GlUe/+N566y0nXuUUljRw5oT6zJn+A6R1umDBgnz/2060z96sdekPS++9916Oj6EuQ6PPUow5ePCg/PPPP9KoUSN3m23bZn3x4sWS7NLS0uTWW2+VO++8U1599VXTXKy0zjIyMoLqrWrVqlKuXDm33vS+Ro0aQU33TZs2NRNvarO8WrJkSdBzKG3yT+S637Bhgznl0LhxY3db0aJFzamHwLorVqyYHH/88e4xWk96Om7p0qXuMQ0aNJDU1NSgulu3bp055eQ/JlT9ar0nGj3VoaeK+vTpY04P7dy5091HfeYsPT3d3BcvXjxf/7YT8bM3a136ff/993LzzTfLvffeK//73/9k37597j7qMjT6LMWYHTt2iM/nC/qjV7quXzrJ7IQTTpA77rjDnH/XfiLaf+mxxx6TF1980XzZ65e0fqEHOu6448w+pfdZ61X3+/f57/3bAo/RD13tN6H9ABKN/2cP9XMH1ov2QQqkfUH0QzjwmAoVKgQd469v3ec/NtzrJAr9otY+XVofGvBHjx4tzzzzjDz99NPmC5j6DE0/+4YPHy7169c34Ufl19+2BvpE+uwNVZeqZcuWJmiWKVPG9DEcNWqU+fm0352iLkMjLCFuNGvWzF3Wjpf+8KQdhhMxxCB+nX322e6yflHp+7V3796mg3HW1g0cphcHaEvQk08+SbVEqS61M3fge7N06dLmGA31esEFQuM0XIzR/979/3kGCpX2k53+p6mtTPpHrnWjzeh6JVeg7du3u/Wm91nrVff79/nv/dsCjylSpEjCBjL/zx7q5w6sF231DKSnRvS/8XD1618/Uv0m+nu7YsWKUqJECfNeVdRn6C/3WbNmSf/+/c2Van759bedSJ+9OdVlKP4r/QLfm9RldoSlGKPNzXXq1JH58+cHNafqul4GisP27t3rBiWtMz0tNG/ePHe/Ni1rnyZ/ven9qlWrgj4w586daz4sq1WrZta1tSrwOfzHJHLd66kircPAn1v7OmhfpMC60y8r7dPhp+9JvUjE/2Grx/z999/miy2w7jTQ+vtM6DGh6lfrPZFt3rzZBEv9L15Rn4fpe0i/3PWSdz2tnvVUbn79bSfCZ++R6jKUFStWmPvA9yZ1mR1hKQa1a9fOjHfzzTffyJo1a+Sdd94xHfDifayfY6Xjo2inWe2QvGjRInnhhRfMf4J6Dl47JJ9//vnmGP1w0y/1N954w/zh+z/otDOnfnAOHjzYfED8+eefMmbMGLn44ovNTOXqoosuMs8/cuRIM1bJ9OnTzWm+tm3bSrwHS/2Z/R+M+jPqsn7haCftNm3ayEcffSS///67+aDUOtIPz9NOO80cr/Wm/XDeeustE6IWLlxoxlxq0aKF6fug9PegXzhDhgwxzf86Js7UqVPN+9lPX2fOnDlmHButXx3bScdk0vGYEqU+dd8HH3xgOsrqdv2Cfv75580pDn0PKurzMP1y1w7H2hFew422auhN+xGp/PzbjvfP3iPVpf5zqX09tQ61LvTvXccE0wsz9FSxoi5Ds/SSuBz2wUM6MJoOeqdv9Fq1apnB/xL9v+8j0cEnteVCryrSJnMdOLFLly7ueXb/wHU//vijad0INXCdDoSoH4Dad6RQoUJm4DodFDHrwHUjRowwH5aJMiil/kxPPPFEtu368/fq1csdlPKLL74wrUpat3q1TOBgdtoyoh/GgYNS6oB9OQ1KqaedNAR16NAh6DX1C0q/yPR3oYNexuOglOHqs0ePHibI64CV2hqnYVKvNLz66quD3ovUZ6bOnTuHrGPtj+j/u8vPv+14/uw9Ul1qmH/ttdfMPzMaArUOdDDVK664woRSP+oyO8ISAABAGJyGAwAACIOwBAAAEAZhCQAAIAzCEgAAQBiEJQAAgDAISwAAAGEQlgAAAMIgLAEAAIRBWAKQUHT6Bh2VPD/pyNA6erLeA0g8hCUAiHE6HY1Ou/H555+bdZ1KRcOZzu8FIPpS8+E1ACDf3HrrrSZcJJL169ebeeb8c5TpJL3HHXdcrmaVB3DsCEsAEkpqauJ9rC1dutRMWFyjRg03LMXL5K5AIki8TxUAcWXLli0yZswYmT17tmk9qVSpkrRr107OP/989xjtC/TEE0/IXXfdJStWrJCvv/5a9u7dKyeffLLcfPPNUq5cuaA+S3/99Ze599PZ6nUmeW2hsSzLHH/BBRdImzZt3GP+/fdfGTVqlMybN08OHDggNWvWNLPSN2/ePKi8mzdvlmHDhpnjdHb7li1bStOmTUP+bEuWLJFx48aZcJORkSHHH3+8XHPNNXLiiScesV7059u/f79ZXrhwoQlKu3btMuv6fGeddZbs2LFDbNuW4sWLH1WdAzg6lpNo7dUA4sa2bdukb9++ZlnDS8mSJeXPP/+U33//XW644QZp27ZtUFjSwKBh57zzzjNB4dNPPzXB54UXXpCCBQuGDEtz586VAQMGSKNGjeT0008329auXWte+5577nHLcf/995twcumll5rw8e2338qqVavk3nvvdR+n+/W4TZs2mePKlCkj3333nQlCK1eulP79+0vDhg3NsfPnz5dnnnlG6tSpI2eeeaYp9zfffGNe+8knn5S6deuGrRstv5bhSMqXLx8UDAFEHi1LADyjLUo+n0/++9//SokSJcy2iy66SF5++WUZP368XHjhhW4IUtqyMmjQIClSpIhZr127tln/4osvglqJAs2aNcsc//DDD5tWmFA+/vhj2b59uwkx/laf1q1by3333ScjRoyQU0891TxWX0dbp+6++27TsuMPeRqgAun/oEOHDjXBqV+/fiYoKf15NKDpz/3II4+ErZv27dvLOeecI+np6fLSSy/J9ddfb8KihjANiQ8++KA5LrB+AEQHV8MB8IQGil9++UVOOeUUs6wtRf6bntbSkPDPP/8EPebcc891g5LSFpvSpUubU3g5KVq0qOzbt8+0MOVEH68tPYGnx7SPkAamjRs3ypo1a9zj9PX0df30VJweF0hPFWqo0lN0O3fudH8u/6nDv//+24TEcKpVqyaNGzc2fbBSUlLMa+j6nj17pF69emZZb7k5pQfg2NCyBMATGh60j5K21ugtp2MCVa5cOWhdW2y0j5MGmpxcfPHFMnPmTHNKTE+bacBo0aJFUD8jPa0WqsN01apV3f3aqqOvo6/nbynyq1KlStC6BiUV7vSYhsGc+hppuNOb0tOStWrVMqcA9aYtS9qPyl83euoSQHQRlgB4wt9dUk81aR+kULST9bHSS+y1T5OGDr1p65D2HdJWqjvvvFOi+bNde+21JuiEoi1XOZk0aZJMmDAhaNstt9ziLmu/p8mTJ5tl7UAOILoISwA8oS0iekpNT0dpa09u+FtsAkNJWlqae0l9TvRUlvY70pu+3jvvvGNas6688krTUqSdxNetW5ftcRpKlP9qO+1MrZ2+9XUDW5eyPrZixYruKcDc/myBNDzq6TVtSXr++efd0KWn7zRIaX+lrK1bAKKHPksAPKEdps844wzTb0kDyJFOwSm98kz77Pj9/PPPsnXrVmnWrFmOr6N9hrK+rr/FSocIUPp4HctIL8n30/5FX375pQlI2n/If5y+nr6un54uy3oaUa+A08CkrT/6PLn52QLpYzVkaeuThiIdRkHXDx48aDq1N2nSxO2zBCD6aFkC4JmuXbuaYQH0SjW9qkxDiV7xph27dRyj9957L+h47ePz2GOPSatWrczVa3pVmLYM6WNzMmTIEPOc2rG6bNmypt/RtGnTTEuNv09Shw4dzFhM2q8pcOgAnU5Ehw7wX0Wnr6OPHTx4sCmjdvbWAKedvAPp8bfddpt5Pr36Tcur/aV0TCn9ebVF7aGHHjpi/SxatMj0h/L3bdIwp527AeQvwhIAz5QqVcoECu2foy1M06dPN0MIVK9eXbp165bt+I4dO5rxjPRSf21h0rGTtC9P1rASSPtEaQvRjBkzTIdyfU297F/nVvOHIN2mYzHpoJQahvT0l7Y+6emuwEEp9XU0rL377rvmOL1sX59fO4vrzxFIhw14+umnzc+mP5e2MOnr6FV3OoRAbmhYql+/vlnWVqVly5aZMAcgfzEoJYCY5x+UUltpAi/bB4D8QJ8lAACAMAhLAAAAYRCWAAAAwqDPEgAAQBi0LAEAAIRBWAIAAAiDsAQAABAGYQkAACAMwhIAAEAYhCUAAIAwCEsAAABhEJYAAAAkZ/8PP5Nm60Cstl4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "moving_avg = np.convolve(episode_rewards, np.ones((SHOW_EVERY,))/SHOW_EVERY,mode='valid')\n",
    "plt.plot([i for i in range(len(moving_avg))], moving_avg)\n",
    "plt.xlabel('episode #')\n",
    "plt.ylabel(f'mean{SHOW_EVERY} reward')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e72299ae",
   "metadata": {},
   "outputs": [
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n",
      "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n",
      "\u001b[1;31mClick <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. \n",
      "\u001b[1;31mView Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "with open(f'qtable_{int(time.time())}.pickle','wb') as f:\n",
    "    pickle.dump(Q_table,f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
